# -*- coding: UTF-8 -*-
"""
@author:vegazhang
@time:2021/2/5
"""
import os
import re

import jsonpath
import pypinyin
import scrapy
from cssselect import Selector

from configs.logging_settings import LogStateString
from extractors.base_extractor import BaseExtractor
from spidertools.utils.xpath_utils import get_alltext, get_all_tables
from spidertools.utils.text_utils import replace_punctuation
from spidertools.utils.snippets import combine_two_dict
from info_fsm import InfoMachine
from table_info_extract.common_table_utils import common_table_extrator
from table_info_extract import dict_mapping_triedTree
from pprint import pprint
from scrapy import Selector

from utils.xpath_utils import get_alltext_exclude_table

from utils.logger_utils import savelog

class ShanDongShengCaiGouYuZhaoBiaoWangqiu(BaseExtractor):
    def __init__(self, info_dict):
        super(ShanDongShengCaiGouYuZhaoBiaoWangqiu, self).__init__(info_dict,one_line_mode_return_result=False)
        self.check_parse_dict = [
            '采购', '补充公告', "废标", "流标", "设计", "终止", "监理", "更正",
            ['不足三家'], ['采 购 人'],
            ["采购单位", "采购项目"],
            ['废标原因', '废标依据'],
            '开标时间变更通知'
        ]


    def check_purchase(self,check_text=None):
        no_need_parse = False

        if check_text == None:
            check_text = self.html
        announcement_title  = self.info_dict['公告标题']

        if self.info_dict.get("工程类型", "") in ['货物', '服务']:
            no_need_parse = True
        else:
            for purchase_obj in self.check_parse_dict:
                if isinstance(purchase_obj, str) and purchase_obj in announcement_title:
                    no_need_parse = True
                elif type(purchase_obj) == list:
                    if not [False for obj in purchase_obj if obj not in check_text]:
                        no_need_parse = True
        return no_need_parse

    def parse_announcement_info(self):

        ########  正文文本解析
        content_path_list = [
            '//div[@class="details"]/div/*',
            '//div[@class="details"]/*',
        ]
        result = None
        for content_xpath in content_path_list:
            text_lists = []
            p_nodes = self.sel.xpath(content_xpath)

            new_p_nodes = []
            for p in p_nodes:
                if p.root.tag == "ol":
                    li_p_nodes = p.xpath('./li/p')
                    new_p_nodes.extend(li_p_nodes)
                elif p.root.tag == 'table':
                    continue
                else:
                    new_p_nodes.append(p)

            for p in new_p_nodes:
                br_nodes = p.xpath('./br')
                if br_nodes:
                    p_html = p.extract()
                    p_html = p_html.replace("<p>","").replace("</p>","")
                    text_lists.extend(p_html.split("<br>"))
                else:
                    text_lists.append(get_alltext_exclude_table(p))
            result = self.texts_matchine_parse(text_lists)

            if result:
                break

        ########  正文table转文本解析
        table_path_content_list = [
            '//div[@class="details"]/div/table/tbody/tr/td',
            '//div[@class="details"]/table/tbody/tr/td',
        ]
        for table_xpath in table_path_content_list:
            text_lists = []
            td_nodes = self.sel.xpath(table_xpath)
            for td_node in td_nodes:
                br_nodes = p.xpath('./br')
                if br_nodes:
                    p_html = p.extract()
                    p_html = p_html.replace("<td>", "").replace("</td>", "").replace("<p>","").replace("</p>","")
                    text_lists.extend(p_html.split("<br>"))
                else:
                    text_lists.append(get_alltext_exclude_table(p))
            table_result = self.texts_matchine_parse(text_lists)
            if table_result:
                result = combine_two_dict(result,table_result)

        ########  正文table解析
        check_list = [
            ["招标人","招标代理机构"], #临清市聚和兴城建设项目供配电工程
            ["项目编号","项目名称"], #京沪高速公路临沂北连接线改建工程公开招标公告
        ]
        tables = get_all_tables(self.sel)
        for table in tables:
            table_text = get_alltext(table)
            all_in = True
            for check in check_list:
                all_in = True
                for item in check:
                    if item not in table_text :
                        all_in = False
                        break
                if all_in:
                    break
            if not all_in:
                continue
            table_result = self.table_extract_machine.run_table(table)
            result = combine_two_dict(result, table_result)

        return result

    def get_infos_from_otherway(self, result, html=None):
        '''
        对提取结果进行一些变换；比如没有项目名称，需要从标段名称里面提取
        或者从标题里面获取项目名称
        '''
        if "工程基本信息" in result and "项目名称" in result["工程基本信息"]:
            if '工程招标信息' in result and '标段名称' not in result['工程招标信息']:
                result['工程招标信息']['标段名称'] = result["工程基本信息"]['项目名称']

        if self.info_dict['公告类型'] == "招标公告":
            if '工程投标信息' in result:
                del result['工程投标信息']
        return result

    def clean_dict_keys(self, result):
        '''
        对result里面的一些字段进行简单的处理，比如清洗掉无用字符等
        '''
        if '工程基本信息' in result and '项目编号' in result['工程基本信息']:
            result['工程基本信息']['项目编号'] = re.sub(r'\),.*',"", result['工程基本信息']['项目编号'])
        pass

        return result

    def common_replace_html(self):
        '''
        对网页源码进行一些特殊处理，并需要更新self.sel对象
        '''


        #长深-京沪高速公路临沂北连接线改建工程公开招标公告 里面的 联系人表格 特殊处理

        pass




class ShanDongShengCaiGouYuZhaoBiaoWang(BaseExtractor):
    def __init__(self, info_dict):
        super(ShanDongShengCaiGouYuZhaoBiaoWang, self).__init__(info_dict)
        self.check_parse_dict = [
            '采购', '补充公告', "废标", "流标", "设计", "终止", "监理", "更正","资格预审",
            ['不足三家'], ['采 购 人'],
            ["采购单位", "采购项目"],
            ['废标原因', '废标依据']
        ]
        self.zhaobiaogonggao_repls_in_html = {
            "发布时间：": "",
        }
        self.zhaobiaogonggao_repls_in_text_context = {

        }
        self.zhongbiaogonggao_repls_in_html = {
            "发布时间：": "", "（项目编号：": "",
            "><p>名称</p>": "><p>中标单位名称</p>",
            "<p>单位名称</p>": "<p>中标单位名称</p>",
        }
        self.zhongbiaogonggao_repls_in_text_context = {
            "公示时间": "\n公示时间",
            "如对评定结果有异议": "\n如对评定结果有异议",
            "异议人不得以异议为名排挤竞争对手": "\n异议人不得以异议为名排挤竞争对手",
        }
        self.check_parse_table_lists = [
            "项目负责人类似业绩", "废标依据", "业绩荣誉得分", "技术部分、人员状况得分"
        ]
        self.get_re_pattern_words = [
            "确定(.+?)为第一中标候选人"
        ]
        self.unit_lists = ['（万元）', "(万元）", "（包）", "(元）", "（元）", "（元)", "(元)","（开始、截止时间）", "（合同编号）"]

    def check_parse(self, check_text=None, announcement_title=None):
        if self.info_dict.get("工程类型", "") in ['货物', '服务', '---']:
            return True
        for purchase_obj in self.check_parse_dict:
            if isinstance(purchase_obj, str) and purchase_obj in announcement_title:
                return True
            elif type(purchase_obj) == list:
                if not [False for obj in purchase_obj if obj not in check_text]:
                    return True
        return False

    def common_replace_word_vega(self, text, sub_dict=None, trans_obj=False, unit=False):  # 一般直接在源网页中改，即self.html
        """
        :param text: 待替换文本
        :param sub_dict: 传入待替换的对应字典
        :param trans_obj: 是否包装为self.sel
        :return: 只替换的话返回文本，否则不返回
        """
        if not sub_dict:
            return text
        # if unit:
        #     for i in self.unit_lists:
        #         for j in range(len(i)):
        for i in sub_dict:
            text = re.sub(i, sub_dict[i], text)
        if trans_obj:
            self.sel = Selector(text=text)
        return text

    def clean_table_text(self, text_all, table_xpath):
        table_text = self.sel.xpath(table_xpath).extract()
        text_pure_text = list(set(text_all).difference(set(table_text)))
        return text_pure_text

    def parse_zhongbiao(self):
        """
        中标公告信息提取
        :return:
        """
        table_parse = True
        tables = get_all_tables(self.sel)
        text_all = self.sel.xpath('//div[@class="details"]//p/text()').extract()
        if tables:
            text_all = self.clean_table_text(text_all, "//table//p/text()")

        text_all = self.common_replace_word_vega("".join(text_all), eval(
            f"self.{self.pinyin(self.info_dict['公告类型'])}_repls_in_text_context"))
        output = self.texts_matchine_parse([text_all])

        if table_parse:
            table_output = {}
            for table in tables:
                f = table.get()
                if self.check_parse_table(table.get()):
                    continue
                result = self.table_extract_machine.run_table(table)
                table_output = combine_two_dict(table_output, result)
            output = combine_two_dict(table_output, output)

        return output

    def check_parse_table(self, text):
        if not self.check_parse_table_lists:
            return None
        for i in self.check_parse_table_lists:
            if i in text:
                return True

    def parse_zhaobiao(self):
        """
        招标公告信息提取
        :return:
        """
        return ''

    def get_jsonpath(self, json_path, result):
        """
            "$..[中标信息名称]"
        """
        return jsonpath.jsonpath(result, json_path)

    def get_fyi(self, result, html, announce_type=None):
        if not jsonpath.jsonpath(result, "$.[工程基本信息]"):
            result["工程基本信息"] = {}

        if not jsonpath.jsonpath(result, "$.[项目名称]"):
            result['工程基本信息']['项目名称'] = self.info_dict['公告标题']

        if announce_type in ["招标公告/资审公告", '招标公告']:
            """招标信息处理"""
            del result["工程投标信息"]
        elif announce_type in ["中标公告"]:
            """中标信息处理"""
            if not jsonpath.jsonpath(result, "$.[工程投标信息]"):
                result["工程投标信息"] = [{}]

            biaoduan_name = jsonpath.jsonpath(result, "$..[标段名称]")
            if (biaoduan_name and not biaoduan_name[0]) or not biaoduan_name:
                result["工程投标信息"][0]['标段名称'] = jsonpath.jsonpath(result, "$..[项目名称]")[0]

            if not jsonpath.jsonpath(result, "$..[中标信息名称]"):
                result = self.get_re_pattern(result)

        return result

    def get_re_pattern(self, result):
        for word in self.get_re_pattern_words:
            re_result = re.findall(f"{word}", self.html)
            if re_result:
                result['工程投标信息'][0]['中标信息名称'] = re_result[0]
        return result

    def pinyin(self, word, upper_first_char=False):
        """
        汉字转拼音
        :param word: 待转换word
        :param upper_first_char: 拼音第一个字符是否大写
        """
        s = ''
        for i in pypinyin.lazy_pinyin(word, style=pypinyin.NORMAL):
            if upper_first_char:
                s += ''.join(i.capitalize())
            else:
                s += ''.join(i)
        return s

    def specially_deal_tem_can_not_parse(self):
        if "下表" in self.html and not self.sel.xpath("//table"):
            return True
        for i in ["名称中标单位项目经理", "序号投标单位名称中标候选人", "中标候选人名称项目负责人预中标价"]:
            if i in self.html:
                return True

    def start_parse(self):
        announce_type = self.info_dict["公告类型"]
        result = {}
        # step1:是否必要解析
        if self.specially_deal_tem_can_not_parse():
            """本网站特殊"""
            savelog(LogStateString.Warning_TempCantParse, self.info_dict['origin_url'])
            return {"not_need_checkjson": '1'}
        if self.check_parse(self.html, self.info_dict['公告标题']):
            savelog(LogStateString.Waring_NoNeedParse, msg_dict=self.info_dict['origin_url'])
            return {"not_need_checkjson": '1'}

        # step2:正则替换
        self.html = self.common_replace_word_vega(self.html, eval(f"self.{self.pinyin(announce_type)}_repls_in_html"),
                                                  trans_obj=True, unit=True)
        # self.sel = Selector(text=self.html)

        # step3: 分类处理
        if announce_type in ['招标公告']:
            result = self.parse_zhaobiao()
        elif announce_type in ["中标公告"]:
            result = self.parse_zhongbiao()

        # step4:特殊处理未获取到的信息
        result = self.get_fyi(result=result, html=self.html, announce_type=announce_type)

        # step5:简单清洗
        result = self.clean_dict_keys(result)
        return result


# class S:
#     def __init__(self):
#         self.zhongbiaogonggao_a = "aaaa"
#
#     def pinyin(self, word, upper_first_char=False):
#         '''
#         汉字转拼音
#         :param word: 待转换word
#         :param upper_first_char: 拼音第一个字符是否大写
#         :return:
#         '''
#         s = ''
#         for i in pypinyin.lazy_pinyin(word, style=pypinyin.NORMAL):
#             if upper_first_char:
#                 s += ''.join(i.capitalize())
#             else:
#                 s += ''.join(i)
#         return s
#     def test(self):
#         print(eval(f"self.{self.pinyin('中标公告')}_a"))
# if __name__ == '__main__':
#     a = S()
#     print(a.test())

if __name__ == '__main__':
    a = {
        "工程基本信息": {
            "项目编号": "2020TDZL18Z0467",
            "项目名称": "平阴县东阮二村等3村提质改造项目",
            "建筑面积": "863678.00平方米",
            "工程类别": "工程",
            "工程所属省份": "山东"
        },
        "工程公告信息": {
            "工程公告信息发布时间": "2020-12-01",
            "工程公告信息结束时间": "2020年12月03日",
            "公告标题": "平阴县东阮二村等3村提质改造项目监理、施工中标结果公告",
            "公告类型": "中标公告"
        },
        "工程招标信息": {
            "招标人信息名称": [
                "平阴县土地整治投资有限公司"
            ],
            "标段名称": "平阴县东阮二村等3村提质改造项目监理、施工",
            "招标方式": "公开招标"
        },
        "工程投标信息": [{}],
        "未知": {
            "未知发布时间": "2020-12-01       阅读量:387"
        }
    }
    print(jsonpath.jsonpath(a, "$..[中标信息名称]"))
    print(jsonpath.jsonpath(a, "$..[标段名称]"))
    print(jsonpath.jsonpath(a, "$.[工程投标信息]"))
    print(jsonpath.jsonpath(a, "$..[项目名称]"))
